{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(style='dark')\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor,BaggingRegressor, StackingRegressor\n",
    "from lightgbm import LGBMRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from catboost import CatBoostRegressor\n",
    "\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "le = LabelEncoder()\n",
    "\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_squared_log_error, r2_score\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(path + \"\\\\Train.csv\")\n",
    "test = pd.read_csv(path+\"\\\\Test.csv\")\n",
    "sample_sub = pd.read_csv(path+\"\\\\sample_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['TARGET(PRICE_IN_LACS)']=np.log1p(train['TARGET(PRICE_IN_LACS)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(98171, 12)"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([train, test], axis = 0).reset_index(drop=True)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SQUARE_FT'] = np.log(df['SQUARE_FT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['POSTED_BY'].replace( {'Owner' : 0, 'Builder' : 1, 'Dealer' : 2}, inplace = True)\n",
    "df['BHK_OR_RK'].replace( {'BHK':1, 'RK': 0}, inplace = True)\n",
    "\n",
    "df['BHK_NO.'].replace( {16:15, 17:15, 18:15, 31:20, 13:11}, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['POSTED_BY'] = df['POSTED_BY'].astype('uint8')\n",
    "df['UNDER_CONSTRUCTION'] = df['UNDER_CONSTRUCTION'].astype('bool')\n",
    "df['RERA'] = df['RERA'].astype('bool')\n",
    "df['BHK_NO.'] = df['BHK_NO.'].astype('int8')\n",
    "df['BHK_OR_RK'] = df['BHK_OR_RK'].astype('bool')\n",
    "df['READY_TO_MOVE'] = df['READY_TO_MOVE'].astype('bool')\n",
    "df['RESALE'] = df['RESALE'].astype('bool')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Extracting Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['City'] = df['ADDRESS'].apply(lambda x: x.split(',')[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Address'] = df['ADDRESS'].apply(lambda x: x.split(',')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('ADDRESS', axis = 1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Grouping Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['median_sqft_per_bhkno'] = df.groupby('BHK_NO.')['SQUARE_FT'].transform('median')\n",
    "df['min_sqft_per_bhkno'] = df.groupby('BHK_NO.')['SQUARE_FT'].transform('min')\n",
    "df['max_sqft_per_bhkno'] = df.groupby('BHK_NO.')['SQUARE_FT'].transform('max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['mean_sqft_per_location'] = df.groupby('LONGITUDE')['SQUARE_FT'].transform('median')\n",
    "df['min_sqft_per_location'] = df.groupby('LONGITUDE')['SQUARE_FT'].transform('min')\n",
    "df['max_sqft_per_location'] = df.groupby('LONGITUDE')['SQUARE_FT'].transform('max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['median_sqft_per_city'] = df.groupby('City')['SQUARE_FT'].transform('median')\n",
    "df['min_sqft_per_city'] = df.groupby('City')['SQUARE_FT'].transform('min')\n",
    "df['max_sqft_per_city'] = df.groupby('City')['SQUARE_FT'].transform('max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['median_sqft_per_address'] = df.groupby('Address')['SQUARE_FT'].transform('median')\n",
    "df['min_sqft_per_address'] = df.groupby('Address')['SQUARE_FT'].transform('min')\n",
    "df['max_sqft_per_address'] = df.groupby('Address')['SQUARE_FT'].transform('max')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sqft_per_room'] = df.apply(lambda x: x['SQUARE_FT']/x['BHK_NO.'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### Price per SQFT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['City'] = train['ADDRESS'].apply(lambda x: x.split(',')[-1])\n",
    "test['City'] = test['ADDRESS'].apply(lambda x: x.split(',')[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array(train['City'].unique())\n",
    "b = np.array(test['City'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "city_replace = {'Alappuzha' : 'Kochi','Amreli' : 'Rajkot','Azamgarh' : 'Varanasi','Barmer' : 'Jodhpur','Barnala' : 'Ludhiana',\n",
    "                'Bellary': 'Anantapur','Bhilwara' : 'Ajmer','Bhusawal' : 'Dhule','Birbhum' : 'Bardhaman','Bulandshahr' : 'Hapur',\n",
    "                'Burhanpur' : 'Jalgaon','Chittorgarh' : 'Udaipur','Contai' : 'Kharagpur','Dewas' : 'Indore','Dhar' : 'Indore',\n",
    "                'Dhenkanal' : 'Cuttack','Dindigul' : 'Madurai','Firozabad' : 'Agra','Gangtok' : 'Darjeeling','Gorakhpur' : 'Varanasi','Gudivada' : 'Vijayawada',\n",
    "                'Jaisalmer' : 'Jaipur','Jalpaiguri' : 'Siliguri','Jhajjar' : 'Rohtak','Jorhat' : 'Nagaon','Kaithal' : 'Patiala','Karimnagar' : 'Warangal',\n",
    "                'Karur' : 'Salem','Katni' : 'Jabalpur','Kaushambi' : 'Satna','Khandwa' : 'Jalgaon','Kolar' : 'Vellore','Malegaon' : 'Dhule',\n",
    "                'Mancherial' : 'Warangal','Mandsaur' : 'Ujjain','Morena' : 'Gwalior','Namakkal' : 'Salem','Palani' : 'Madurai','Panchmahal' : 'Godhra',\n",
    "                'Patan' : 'Gandhinagar','Pathankot' : 'Jammu','Pollachi' : 'Coimbatore','Purulia' : 'Dhanbad','Raebareli' : 'Lucknow','Rajsamand' : 'Udaipur',\n",
    "                'Ramgarh' : 'Nainital','Rampur' : 'Moradabad','Ratlam' : 'Ujjain','Rupnagar' : 'Chandigarh','Shimoga' : 'Mangalore','Sirsa' : 'Patiala',\n",
    "                'Sivasagar' : 'Dibrugarh','Tezpur' : 'Guwahati','Theni' : 'Madurai','Thiruvarur' : 'Thanjavur','Tiruchengode' : 'Erode',\n",
    "                'Tumkur' : 'Bangalore','Viramgam' : 'Gandhinagar','Wayanad' : 'Ooty'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_list = list(np.setdiff1d(b,a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['price_per_sqft'] = train['TARGET(PRICE_IN_LACS)']/train['SQUARE_FT']\n",
    "\n",
    "mean_price_per_sqft = train.groupby('City')['price_per_sqft'].mean().to_dict()\n",
    "median_price_per_sqft = train.groupby('City')['price_per_sqft'].median().to_dict()\n",
    "min_price_per_sqft = train.groupby('City')['price_per_sqft'].min().to_dict()\n",
    "max_price_per_sqft = train.groupby('City')['price_per_sqft'].max().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "def price_per_sqft_imputer(x, dictionary):\n",
    "    if x in replace_list:\n",
    "        return -1 #dictionary[city_replace[x]]\n",
    "    return dictionary[x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['mean_price_per_sqft'] = df['City'].apply(lambda x : price_per_sqft_imputer(x, mean_price_per_sqft))\n",
    "df['median_price_per_sqft'] = df['City'].apply(lambda x : price_per_sqft_imputer(x, median_price_per_sqft))\n",
    "\n",
    "df['min_price_per_sqft'] = df['City'].apply(lambda x : price_per_sqft_imputer(x, min_price_per_sqft))\n",
    "df['max_price_per_sqft'] = df['City'].apply(lambda x : price_per_sqft_imputer(x, max_price_per_sqft))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "median_price_per_city_dict = train.groupby('City')['TARGET(PRICE_IN_LACS)'].median().to_dict()\n",
    "min_price_per_city_dict = train.groupby('City')['TARGET(PRICE_IN_LACS)'].min().to_dict()\n",
    "max_price_per_city_dict = train.groupby('City')['TARGET(PRICE_IN_LACS)'].max().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['med_price_per_city'] = df['City'].apply(lambda x : median_price_per_city_dict[ city_replace[x]] if x in replace_list else median_price_per_city_dict[x])\n",
    "df['min_price_per_city'] = df['City'].apply(lambda x : min_price_per_city_dict[ city_replace[x]] if x in replace_list else min_price_per_city_dict[x])\n",
    "df['max_price_per_city'] = df['City'].apply(lambda x : max_price_per_city_dict[ city_replace[x]] if x in replace_list else max_price_per_city_dict[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### Preprocessing New Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "col_ls = ['SQUARE_FT', 'LONGITUDE', 'LATITUDE','median_sqft_per_bhkno',\n",
    "       'min_sqft_per_bhkno', 'max_sqft_per_bhkno', 'mean_sqft_per_location',\n",
    "       'min_sqft_per_location', 'max_sqft_per_location',\n",
    "       'median_sqft_per_city', 'min_sqft_per_city', 'max_sqft_per_city',\n",
    "       'median_sqft_per_address', 'min_sqft_per_address',\n",
    "       'max_sqft_per_address', 'sqft_per_room', 'mean_price_per_sqft',\n",
    "       'median_price_per_sqft', 'min_price_per_sqft', 'max_price_per_sqft',\n",
    "       'med_price_per_city', 'min_price_per_city', 'max_price_per_city']\n",
    "\n",
    "\n",
    "for col in col_ls:\n",
    "    df[col] = df[col].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[ 'City', 'Address']] = df[[ 'City', 'Address']].apply(le.fit_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_proc, test_proc = df[:train.shape[0]], df[train.shape[0]:].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = 'TARGET(PRICE_IN_LACS)'\n",
    "\n",
    "\n",
    "features = [col for col in df.columns if col not in ([target])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn, val = train_test_split(train_proc, test_size = 0.2, random_state = 1999)\n",
    "\n",
    "##### Input for model\n",
    "X_trn, X_val = trn[features], val[features]\n",
    "\n",
    "##### Target column\n",
    "y_trn, y_val = trn[target], val[target]\n",
    "\n",
    "##### Features for test data that we will be predicting\n",
    "X_test = test_proc[features]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_squared_log_error is : 0.29811099376619293\n",
      "Wall time: 523 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lgb = LGBMRegressor(random_state=1999)\n",
    "\n",
    "lgb.fit(X_trn, y_trn)\n",
    "\n",
    "preds = lgb.predict(X_val)\n",
    "preds = np.abs(preds)\n",
    "\n",
    "error = np.sqrt(mean_squared_error(y_val, preds))\n",
    "print(f'mean_squared_log_error is : {error}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_squared_log_error is : 0.29412312577766475\n",
      "Wall time: 1.41 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "xgb = XGBRegressor()\n",
    "\n",
    "xgb.fit(X_trn, y_trn)\n",
    "preds = xgb.predict(X_val)\n",
    "preds = np.abs(preds)\n",
    "\n",
    "error = np.sqrt(mean_squared_error(y_val, preds))\n",
    "\n",
    "print(f'mean_squared_log_error is : {error}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_squared_log_error is : 0.28726922298542\n",
      "Wall time: 3.21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "rf = RandomForestRegressor(random_state = 1999, n_jobs = -1)\n",
    "\n",
    "rf.fit(X_trn, y_trn)\n",
    "preds = rf.predict(X_val)\n",
    "preds = np.abs(preds)\n",
    "\n",
    "error = np.sqrt(mean_squared_error(y_val, preds))\n",
    "\n",
    "print(f'mean_squared_log_error is : {error}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Cross Validation Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross Validation for Boosting\n",
    "def cross_val(regressor, train, test, features, name):\n",
    "    N_splits = 5\n",
    "    \n",
    "    oofs = np.zeros(len(train))\n",
    "    preds = np.zeros(len(test))\n",
    "    \n",
    "    target_col = train[target]\n",
    "    \n",
    "    folds = StratifiedKFold(n_splits = N_splits, shuffle = True,random_state = 1999)\n",
    "    stratified_target = pd.qcut( train[target], 10, labels=False, duplicates='drop')\n",
    "    for index, (trn_idx, val_idx) in enumerate(folds.split(train, stratified_target)):\n",
    "        print(f'\\n================================Fold{index + 1}===================================')\n",
    "        \n",
    "        #### Train Set\n",
    "        X_trn, y_trn = train[features].iloc[trn_idx], train[target].iloc[trn_idx]\n",
    "        \n",
    "        #### Validation Set\n",
    "        X_val, y_val = train[features].iloc[val_idx], train[target].iloc[val_idx]\n",
    "        \n",
    "        #### Test Set\n",
    "        X_test = test[features]\n",
    "        \n",
    "        if name != 'cat':\n",
    "            #### Scaling Data ####\n",
    "            scaler = StandardScaler()\n",
    "            _ = scaler.fit(X_trn)\n",
    "            X_trn = scaler.transform(X_trn)\n",
    "            X_val = scaler.transform(X_val)\n",
    "            X_test = scaler.transform(X_test)\n",
    "        \n",
    "        ############ Fitting #############\n",
    "        _ = regressor.fit(X_trn, y_trn, eval_set = [(X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "        \n",
    "        ############ Predicting #############\n",
    "        val_preds = np.abs(regressor.predict(X_val))\n",
    "        test_preds = np.abs(regressor.predict(X_test))\n",
    "        \n",
    "        error = np.sqrt(mean_squared_error(y_val, val_preds))\n",
    "        print(f'\\n Root Log Mean Squared Error for Validation set is : {error}')\n",
    "        \n",
    "        oofs[val_idx] = val_preds\n",
    "        preds += test_preds / N_splits\n",
    "        \n",
    "    total_error = np.sqrt(mean_squared_error(target_col, oofs))\n",
    "    print(f'\\n\\Root Log Mean Squared Error for oofs is {total_error}')\n",
    "    \n",
    "    return oofs, preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal_cross_val(regressor, train, test, features):\n",
    "    N_splits = 5\n",
    "    \n",
    "    oofs = np.zeros(len(train))\n",
    "    preds = np.zeros(len(test))\n",
    "    \n",
    "    target_col = train[target]\n",
    "    \n",
    "    folds = StratifiedKFold(n_splits = N_splits, shuffle = True,random_state = 1999)\n",
    "    stratified_target = pd.qcut( train[target], 10, labels=False, duplicates='drop')\n",
    "    for index, (trn_idx, val_idx) in enumerate(folds.split(train, stratified_target)):\n",
    "        print(f'\\n================================Fold{index + 1}===================================')\n",
    "        \n",
    "        #### Train Set\n",
    "        X_trn, y_trn = train[features].iloc[trn_idx], train[target].iloc[trn_idx]\n",
    "        \n",
    "        #### Validation Set\n",
    "        X_val, y_val = train[features].iloc[val_idx], train[target].iloc[val_idx]\n",
    "        \n",
    "        #### Test Set\n",
    "        X_test = test[features]\n",
    "        \n",
    "        #### Scaling Data ####\n",
    "        scaler = StandardScaler()\n",
    "        _ = scaler.fit(X_trn)\n",
    "        \n",
    "        X_trn = scaler.transform(X_trn)\n",
    "        X_val = scaler.transform(X_val)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        \n",
    "        ############ Fitting #############\n",
    "        _ = regressor.fit(X_trn, y_trn)\n",
    "        \n",
    "        ############ Predicting #############\n",
    "        val_preds = np.abs(regressor.predict(X_val))\n",
    "        test_preds = np.abs(regressor.predict(X_test))\n",
    "        \n",
    "        error = np.sqrt(mean_squared_error(y_val, val_preds))\n",
    "        print(f'\\n Root Log Mean Squared Error for Validation set is : {error}')\n",
    "        \n",
    "        oofs[val_idx] = val_preds\n",
    "        preds += test_preds / N_splits\n",
    "        \n",
    "    total_error = np.sqrt(mean_squared_error(target_col, oofs))\n",
    "    print(f'\\n\\Root Log Mean Squared Error for oofs is {total_error}')\n",
    "    \n",
    "    return oofs, preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Predicting With Tuned Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1647,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2763880933616468\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.27755309309921083\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.29167454288372224\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.28359457259551735\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2922550468744347\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.2843723863134213\n",
      "Wall time: 3min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rf_oofs, rf_preds = normal_cross_val(rf, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1648,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.26629946870958443\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2693908562460813\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.28204591075507646\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2669893360007582\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2807970014732888\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.273190910022824\n",
      "Wall time: 22.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lgb_oofs, lgb_preds = cross_val(lgb, train_proc, test_proc, features, 'lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1649,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2584373087548216\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.26075218361136987\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.27776801953128816\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.26666927488703795\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.27077540048959825\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.2669710377287559\n",
      "Wall time: 57.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "xgb_oofs, xgb_preds = cross_val(xgb, train_proc, test_proc, features, 'xgb')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna.samplers import TPESampler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:10:10,308] A new study created in memory with name: no-name-ee35fa3a-9a05-4ba2-88a5-c89b769aadab\n",
      "[I 2020-10-24 12:10:14,611] Trial 0 finished with value: 0.5857039572050123 and parameters: {'max_depth': 2, 'n_estimators': 1463, 'min_samples_split': 5, 'max_leaf_nodes': 9325, 'max_features': 0.5988509574287779}. Best is trial 0 with value: 0.5857039572050123.\n",
      "[I 2020-10-24 12:10:24,106] Trial 1 finished with value: 0.2835654202084174 and parameters: {'max_depth': 25, 'n_estimators': 770, 'min_samples_split': 2, 'max_leaf_nodes': 6844, 'max_features': 0.31812503566409056}. Best is trial 1 with value: 0.2835654202084174.\n",
      "[I 2020-10-24 12:10:29,859] Trial 2 finished with value: 0.4949274215231798 and parameters: {'max_depth': 3, 'n_estimators': 1186, 'min_samples_split': 5, 'max_leaf_nodes': 2322, 'max_features': 0.5544356488751458}. Best is trial 1 with value: 0.2835654202084174.\n",
      "[I 2020-10-24 12:10:34,843] Trial 3 finished with value: 0.2958470716175937 and parameters: {'max_depth': 27, 'n_estimators': 772, 'min_samples_split': 3, 'max_leaf_nodes': 6136, 'max_features': 0.11617471795226059}. Best is trial 1 with value: 0.2835654202084174.\n",
      "[I 2020-10-24 12:10:49,619] Trial 4 finished with value: 0.34702086662105025 and parameters: {'max_depth': 7, 'n_estimators': 1131, 'min_samples_split': 2, 'max_leaf_nodes': 8391, 'max_features': 0.739326851373379}. Best is trial 1 with value: 0.2835654202084174.\n",
      "[I 2020-10-24 12:11:25,686] Trial 5 finished with value: 0.2833256234267288 and parameters: {'max_depth': 31, 'n_estimators': 1359, 'min_samples_split': 5, 'max_leaf_nodes': 8715, 'max_features': 0.6119368170620191}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:11:47,566] Trial 6 finished with value: 0.30262877390166903 and parameters: {'max_depth': 11, 'n_estimators': 1397, 'min_samples_split': 5, 'max_leaf_nodes': 8836, 'max_features': 0.5174786574000574}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:12:02,556] Trial 7 finished with value: 0.2919358031621753 and parameters: {'max_depth': 25, 'n_estimators': 1375, 'min_samples_split': 4, 'max_leaf_nodes': 1307, 'max_features': 0.2732402835394975}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:12:05,351] Trial 8 finished with value: 0.6166650916842288 and parameters: {'max_depth': 2, 'n_estimators': 828, 'min_samples_split': 2, 'max_leaf_nodes': 7321, 'max_features': 0.21973989374694655}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:12:23,429] Trial 9 finished with value: 0.28751514948378326 and parameters: {'max_depth': 19, 'n_estimators': 1035, 'min_samples_split': 2, 'max_leaf_nodes': 1741, 'max_features': 0.45995999192898207}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:13:07,393] Trial 10 finished with value: 0.2839888904628947 and parameters: {'max_depth': 34, 'n_estimators': 1286, 'min_samples_split': 4, 'max_leaf_nodes': 4774, 'max_features': 0.8594241151524176}. Best is trial 5 with value: 0.2833256234267288.\n",
      "[I 2020-10-24 12:13:22,889] Trial 11 finished with value: 0.28312920849730494 and parameters: {'max_depth': 35, 'n_estimators': 951, 'min_samples_split': 3, 'max_leaf_nodes': 4592, 'max_features': 0.3921267137687928}. Best is trial 11 with value: 0.28312920849730494.\n",
      "[I 2020-10-24 12:13:48,395] Trial 12 finished with value: 0.2836678451434449 and parameters: {'max_depth': 35, 'n_estimators': 980, 'min_samples_split': 3, 'max_leaf_nodes': 4011, 'max_features': 0.6972459360540051}. Best is trial 11 with value: 0.28312920849730494.\n",
      "[I 2020-10-24 12:14:03,180] Trial 13 finished with value: 0.2834104698233121 and parameters: {'max_depth': 32, 'n_estimators': 889, 'min_samples_split': 3, 'max_leaf_nodes': 3803, 'max_features': 0.4350898164381684}. Best is trial 11 with value: 0.28312920849730494.\n",
      "[I 2020-10-24 12:14:24,132] Trial 14 finished with value: 0.28292037006210186 and parameters: {'max_depth': 29, 'n_estimators': 1265, 'min_samples_split': 4, 'max_leaf_nodes': 5473, 'max_features': 0.38419135349308847}. Best is trial 14 with value: 0.28292037006210186.\n",
      "[I 2020-10-24 12:14:44,820] Trial 15 finished with value: 0.28342548984580856 and parameters: {'max_depth': 19, 'n_estimators': 1226, 'min_samples_split': 4, 'max_leaf_nodes': 5260, 'max_features': 0.377121982703083}. Best is trial 14 with value: 0.28292037006210186.\n",
      "[I 2020-10-24 12:14:51,185] Trial 16 finished with value: 0.29122233977560447 and parameters: {'max_depth': 30, 'n_estimators': 957, 'min_samples_split': 3, 'max_leaf_nodes': 3548, 'max_features': 0.14892076805694227}. Best is trial 14 with value: 0.28292037006210186.\n",
      "[I 2020-10-24 12:15:07,369] Trial 17 finished with value: 0.28497943310595364 and parameters: {'max_depth': 22, 'n_estimators': 1119, 'min_samples_split': 4, 'max_leaf_nodes': 2789, 'max_features': 0.3684639182965956}. Best is trial 14 with value: 0.28292037006210186.\n",
      "[I 2020-10-24 12:15:12,911] Trial 18 finished with value: 0.3359575686896887 and parameters: {'max_depth': 12, 'n_estimators': 1028, 'min_samples_split': 4, 'max_leaf_nodes': 163, 'max_features': 0.21065589093157258}. Best is trial 14 with value: 0.28292037006210186.\n",
      "[I 2020-10-24 12:15:35,809] Trial 19 finished with value: 0.28264188208380153 and parameters: {'max_depth': 35, 'n_estimators': 1269, 'min_samples_split': 3, 'max_leaf_nodes': 5830, 'max_features': 0.39921981803609835}. Best is trial 19 with value: 0.28264188208380153.\n",
      "[I 2020-10-24 12:15:54,159] Trial 20 finished with value: 0.28325800832329534 and parameters: {'max_depth': 28, 'n_estimators': 1309, 'min_samples_split': 3, 'max_leaf_nodes': 5643, 'max_features': 0.30881541447299976}. Best is trial 19 with value: 0.28264188208380153.\n",
      "[I 2020-10-24 12:16:14,805] Trial 21 finished with value: 0.28268381619411836 and parameters: {'max_depth': 35, 'n_estimators': 1207, 'min_samples_split': 3, 'max_leaf_nodes': 6692, 'max_features': 0.3854182610690974}. Best is trial 19 with value: 0.28264188208380153.\n",
      "[I 2020-10-24 12:16:41,252] Trial 22 finished with value: 0.28238393157382563 and parameters: {'max_depth': 33, 'n_estimators': 1228, 'min_samples_split': 3, 'max_leaf_nodes': 7498, 'max_features': 0.4667074940471213}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:17:06,951] Trial 23 finished with value: 0.2828076118692687 and parameters: {'max_depth': 35, 'n_estimators': 1190, 'min_samples_split': 3, 'max_leaf_nodes': 6916, 'max_features': 0.4790427517549887}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:17:42,279] Trial 24 finished with value: 0.28267627796508593 and parameters: {'max_depth': 33, 'n_estimators': 1478, 'min_samples_split': 3, 'max_leaf_nodes': 7657, 'max_features': 0.5321759189768004}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:18:22,846] Trial 25 finished with value: 0.2825916574979586 and parameters: {'max_depth': 32, 'n_estimators': 1489, 'min_samples_split': 2, 'max_leaf_nodes': 7816, 'max_features': 0.6575996342838347}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:19:05,296] Trial 26 finished with value: 0.2826068625451705 and parameters: {'max_depth': 25, 'n_estimators': 1497, 'min_samples_split': 2, 'max_leaf_nodes': 9954, 'max_features': 0.6758360506554477}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:19:51,415] Trial 27 finished with value: 0.28322536009471333 and parameters: {'max_depth': 23, 'n_estimators': 1492, 'min_samples_split': 2, 'max_leaf_nodes': 9653, 'max_features': 0.7205617187894611}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:20:43,506] Trial 28 finished with value: 0.2840036448643396 and parameters: {'max_depth': 26, 'n_estimators': 1442, 'min_samples_split': 2, 'max_leaf_nodes': 7789, 'max_features': 0.845055104538232}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:21:21,753] Trial 29 finished with value: 0.2827477965193291 and parameters: {'max_depth': 22, 'n_estimators': 1422, 'min_samples_split': 2, 'max_leaf_nodes': 9784, 'max_features': 0.6059322409044134}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:22:00,909] Trial 30 finished with value: 0.28258323586896533 and parameters: {'max_depth': 31, 'n_estimators': 1337, 'min_samples_split': 2, 'max_leaf_nodes': 9174, 'max_features': 0.6622605639984025}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:22:39,918] Trial 31 finished with value: 0.2825770685285331 and parameters: {'max_depth': 31, 'n_estimators': 1339, 'min_samples_split': 2, 'max_leaf_nodes': 9211, 'max_features': 0.6616835226149216}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:23:26,774] Trial 32 finished with value: 0.28297697559518437 and parameters: {'max_depth': 31, 'n_estimators': 1330, 'min_samples_split': 2, 'max_leaf_nodes': 9157, 'max_features': 0.7691036030500654}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:24:06,933] Trial 33 finished with value: 0.28258469704440936 and parameters: {'max_depth': 29, 'n_estimators': 1330, 'min_samples_split': 2, 'max_leaf_nodes': 8259, 'max_features': 0.6620829780958792}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:24:51,592] Trial 34 finished with value: 0.28355413022689624 and parameters: {'max_depth': 29, 'n_estimators': 1242, 'min_samples_split': 2, 'max_leaf_nodes': 8134, 'max_features': 0.8153653675053995}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:25:20,571] Trial 35 finished with value: 0.28295617013106433 and parameters: {'max_depth': 27, 'n_estimators': 1155, 'min_samples_split': 2, 'max_leaf_nodes': 9202, 'max_features': 0.5678989231864815}. Best is trial 22 with value: 0.28238393157382563.\n",
      "[I 2020-10-24 12:25:58,477] Trial 36 finished with value: 0.28228399463262116 and parameters: {'max_depth': 30, 'n_estimators': 1328, 'min_samples_split': 2, 'max_leaf_nodes': 8503, 'max_features': 0.6317825761282754}. Best is trial 36 with value: 0.28228399463262116.\n",
      "[I 2020-10-24 12:26:28,013] Trial 37 finished with value: 0.28490447545764475 and parameters: {'max_depth': 16, 'n_estimators': 1392, 'min_samples_split': 2, 'max_leaf_nodes': 8702, 'max_features': 0.5780608671973579}. Best is trial 36 with value: 0.28228399463262116.\n",
      "[I 2020-10-24 12:27:03,087] Trial 38 finished with value: 0.2822750601045757 and parameters: {'max_depth': 31, 'n_estimators': 1317, 'min_samples_split': 2, 'max_leaf_nodes': 6653, 'max_features': 0.6297197869507615}. Best is trial 38 with value: 0.2822750601045757.\n",
      "[I 2020-10-24 12:27:31,116] Trial 39 finished with value: 0.2829291299666294 and parameters: {'max_depth': 33, 'n_estimators': 1150, 'min_samples_split': 2, 'max_leaf_nodes': 6428, 'max_features': 0.6133522830659035}. Best is trial 38 with value: 0.2822750601045757.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized RF RMSLE 0.28227506010457565\n",
      "Wall time: 20min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def create_model(trial):\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 2, 35)\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 700, 1500)\n",
    "    min_samples_split = trial.suggest_int(\"min_samples_split\", 2, 5)\n",
    "    max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 100,10000)\n",
    "    max_features = trial.suggest_uniform('max_features', 0.1, 0.9)\n",
    "    model = RandomForestRegressor( \n",
    "        max_depth=max_depth,\n",
    "        n_estimators = n_estimators,\n",
    "        min_samples_split = min_samples_split,\n",
    "        max_leaf_nodes = max_leaf_nodes,\n",
    "        max_features = max_features,\n",
    "        random_state=1999,\n",
    "        bootstrap = True,\n",
    "        n_jobs = -1\n",
    "    )\n",
    "    return model\n",
    "\n",
    "sampler = TPESampler(seed=0)\n",
    "def objective(trial):\n",
    "    model = create_model(trial)\n",
    "    model.fit(X_trn, y_trn)\n",
    "    preds = model.predict(X_val)\n",
    "    score = np.sqrt(mean_squared_error(y_val,preds))\n",
    "    return score\n",
    "\n",
    "study = optuna.create_study(direction=\"minimize\", sampler=sampler)\n",
    "study.optimize(objective, n_trials=40)\n",
    "\n",
    "rf_params = study.best_params\n",
    "rf_params['random_state'] = 1999\n",
    "rf = RandomForestRegressor(**rf_params)\n",
    "rf.fit(X_trn, y_trn)\n",
    "preds = rf.predict(X_val)\n",
    "print('Optimized RF RMSLE', np.sqrt(mean_squared_error(y_val, preds)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:32:40,194] A new study created in memory with name: no-name-defb7de4-c882-42c0-b173-524036bd1cda\n",
      "[I 2020-10-24 12:32:41,391] Trial 0 finished with value: 0.2954044505998161 and parameters: {'max_depth': 2, 'n_estimators': 1535, 'learning_rate': 0.8721510558604811, 'colsample_bytree': 0.7778013910273004, 'num_leaves': 213, 'reg_alpha': 0.6167152904533248, 'reg_lambda': 0.45006976901015405}. Best is trial 0 with value: 0.2954044505998161.\n",
      "[I 2020-10-24 12:32:42,322] Trial 1 finished with value: 0.2840497715875627 and parameters: {'max_depth': 8, 'n_estimators': 1300, 'learning_rate': 0.3453906651221019, 'colsample_bytree': 0.4821320938570799, 'num_leaves': 41, 'reg_alpha': 0.5231159358023236, 'reg_lambda': 0.5544356488751458}. Best is trial 1 with value: 0.2840497715875627.\n",
      "[I 2020-10-24 12:32:45,209] Trial 2 finished with value: 0.276234921574609 and parameters: {'max_depth': 39, 'n_estimators': 1237, 'learning_rate': 0.16393245237809825, 'colsample_bytree': 0.1697034397612326, 'num_leaves': 117, 'reg_alpha': 0.39459323187243844, 'reg_lambda': 0.8657241271624371}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:46,177] Trial 3 finished with value: 0.3400434984105895 and parameters: {'max_depth': 17, 'n_estimators': 1148, 'learning_rate': 0.9807565080094875, 'colsample_bytree': 0.739326851373379, 'num_leaves': 245, 'reg_alpha': 0.5163819836409639, 'reg_lambda': 0.6431036240951682}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:47,195] Trial 4 finished with value: 0.3097532693183155 and parameters: {'max_depth': 34, 'n_estimators': 1989, 'learning_rate': 0.5836359065041096, 'colsample_bytree': 0.7068924994578858, 'num_leaves': 204, 'reg_alpha': 0.4317295519924189, 'reg_lambda': 0.3116444896837016}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:48,204] Trial 5 finished with value: 0.3009605006149736 and parameters: {'max_depth': 13, 'n_estimators': 1582, 'learning_rate': 0.5105352989948937, 'colsample_bytree': 0.5547471590949188, 'num_leaves': 130, 'reg_alpha': 0.3593128062345713, 'reg_lambda': 0.21973989374694655}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:48,793] Trial 6 finished with value: 0.33849219855488966 and parameters: {'max_depth': 19, 'n_estimators': 1035, 'learning_rate': 0.9123386279764641, 'colsample_bytree': 0.45995999192898207, 'num_leaves': 33, 'reg_alpha': 0.4496255630394732, 'reg_lambda': 0.6581049567418119}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:52,199] Trial 7 finished with value: 0.2852672548991172 and parameters: {'max_depth': 37, 'n_estimators': 1843, 'learning_rate': 0.25381862662244065, 'colsample_bytree': 0.38652173357562003, 'num_leaves': 386, 'reg_alpha': 0.35234268073934716, 'reg_lambda': 0.3909686167540981}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:53,278] Trial 8 finished with value: 0.338982132342564 and parameters: {'max_depth': 22, 'n_estimators': 903, 'learning_rate': 0.9895364542533036, 'colsample_bytree': 0.18163584859842247, 'num_leaves': 129, 'reg_alpha': 0.608047098882851, 'reg_lambda': 0.8962396541423101}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:32:54,577] Trial 9 finished with value: 0.30699000902403245 and parameters: {'max_depth': 16, 'n_estimators': 1867, 'learning_rate': 0.5272277520595716, 'colsample_bytree': 0.5988080809054946, 'num_leaves': 209, 'reg_alpha': 0.1883001129314441, 'reg_lambda': 0.6250636715722188}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:33:01,379] Trial 10 finished with value: 0.2832954091124832 and parameters: {'max_depth': 29, 'n_estimators': 754, 'learning_rate': 0.11034603712143903, 'colsample_bytree': 0.10891410883626335, 'num_leaves': 396, 'reg_alpha': 0.8842659396036715, 'reg_lambda': 0.79893639038804}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:33:10,201] Trial 11 finished with value: 0.2835533667872374 and parameters: {'max_depth': 29, 'n_estimators': 781, 'learning_rate': 0.10711068032220841, 'colsample_bytree': 0.11028536168703411, 'num_leaves': 492, 'reg_alpha': 0.8418365043939721, 'reg_lambda': 0.8929590173621234}. Best is trial 2 with value: 0.276234921574609.\n",
      "[I 2020-10-24 12:33:15,131] Trial 12 finished with value: 0.2735760295505627 and parameters: {'max_depth': 29, 'n_estimators': 775, 'learning_rate': 0.12023004867776622, 'colsample_bytree': 0.2869989357180732, 'num_leaves': 368, 'reg_alpha': 0.8860114658549174, 'reg_lambda': 0.8305769779454338}. Best is trial 12 with value: 0.2735760295505627.\n",
      "[I 2020-10-24 12:33:18,026] Trial 13 finished with value: 0.28967046378419886 and parameters: {'max_depth': 40, 'n_estimators': 1303, 'learning_rate': 0.26864960839497054, 'colsample_bytree': 0.25513928106558736, 'num_leaves': 348, 'reg_alpha': 0.1405887412513422, 'reg_lambda': 0.7825110058588737}. Best is trial 12 with value: 0.2735760295505627.\n",
      "[I 2020-10-24 12:33:24,209] Trial 14 finished with value: 0.2730527748346632 and parameters: {'max_depth': 29, 'n_estimators': 1015, 'learning_rate': 0.13390066922437183, 'colsample_bytree': 0.2933596857107967, 'num_leaves': 322, 'reg_alpha': 0.7474907095269822, 'reg_lambda': 0.7784428529827074}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:26,316] Trial 15 finished with value: 0.29568615211614313 and parameters: {'max_depth': 26, 'n_estimators': 950, 'learning_rate': 0.38416550861436866, 'colsample_bytree': 0.32189182067979577, 'num_leaves': 304, 'reg_alpha': 0.7751013766634733, 'reg_lambda': 0.7578403101074914}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:28,277] Trial 16 finished with value: 0.30761491235386745 and parameters: {'max_depth': 33, 'n_estimators': 706, 'learning_rate': 0.6391566913647807, 'colsample_bytree': 0.3007754199853494, 'num_leaves': 484, 'reg_alpha': 0.7809931244010407, 'reg_lambda': 0.7125395465333261}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:29,903] Trial 17 finished with value: 0.3171489832896667 and parameters: {'max_depth': 24, 'n_estimators': 1055, 'learning_rate': 0.7238382866780985, 'colsample_bytree': 0.8888012003550287, 'num_leaves': 438, 'reg_alpha': 0.6888272305024468, 'reg_lambda': 0.10414372062564015}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:33,138] Trial 18 finished with value: 0.27779879845873545 and parameters: {'max_depth': 29, 'n_estimators': 864, 'learning_rate': 0.20649850612100834, 'colsample_bytree': 0.39331330537951287, 'num_leaves': 321, 'reg_alpha': 0.8754852952052513, 'reg_lambda': 0.5442105374643413}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:34,898] Trial 19 finished with value: 0.2861867721058958 and parameters: {'max_depth': 33, 'n_estimators': 1460, 'learning_rate': 0.37452196681897865, 'colsample_bytree': 0.23119571190237548, 'num_leaves': 285, 'reg_alpha': 0.7227068992186804, 'reg_lambda': 0.8541353697236409}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:41,649] Trial 20 finished with value: 0.2755594209210742 and parameters: {'max_depth': 26, 'n_estimators': 1119, 'learning_rate': 0.10875852319767915, 'colsample_bytree': 0.34705116180416185, 'num_leaves': 434, 'reg_alpha': 0.6439215885611231, 'reg_lambda': 0.7045713810779596}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:46,288] Trial 21 finished with value: 0.27625122462815593 and parameters: {'max_depth': 27, 'n_estimators': 1128, 'learning_rate': 0.13188387156261747, 'colsample_bytree': 0.3725315022737975, 'num_leaves': 427, 'reg_alpha': 0.6422456334465604, 'reg_lambda': 0.6988650936761578}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:48,994] Trial 22 finished with value: 0.2809502793911956 and parameters: {'max_depth': 23, 'n_estimators': 962, 'learning_rate': 0.29088660086247076, 'colsample_bytree': 0.3036267447991441, 'num_leaves': 353, 'reg_alpha': 0.7702686943025954, 'reg_lambda': 0.8023834225629083}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:33:52,968] Trial 23 finished with value: 0.2794211083575777 and parameters: {'max_depth': 32, 'n_estimators': 817, 'learning_rate': 0.19524558948324877, 'colsample_bytree': 0.4272356384036711, 'num_leaves': 448, 'reg_alpha': 0.8237518026017424, 'reg_lambda': 0.7320381056462683}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:02,095] Trial 24 finished with value: 0.2744885910890837 and parameters: {'max_depth': 36, 'n_estimators': 1131, 'learning_rate': 0.10520053218897582, 'colsample_bytree': 0.22183793041335423, 'num_leaves': 374, 'reg_alpha': 0.7000123015481139, 'reg_lambda': 0.5875521536024824}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:04,044] Trial 25 finished with value: 0.2879399523087921 and parameters: {'max_depth': 36, 'n_estimators': 987, 'learning_rate': 0.4402136137381465, 'colsample_bytree': 0.22222385110830034, 'num_leaves': 370, 'reg_alpha': 0.8995298766759203, 'reg_lambda': 0.5936140755551269}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:07,716] Trial 26 finished with value: 0.28261261776997615 and parameters: {'max_depth': 31, 'n_estimators': 700, 'learning_rate': 0.19961555828231436, 'colsample_bytree': 0.1607365233086811, 'num_leaves': 270, 'reg_alpha': 0.6958049409215167, 'reg_lambda': 0.45678919151790937}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:11,314] Trial 27 finished with value: 0.2855482303915259 and parameters: {'max_depth': 37, 'n_estimators': 1221, 'learning_rate': 0.29301458765818655, 'colsample_bytree': 0.26532360138278976, 'num_leaves': 329, 'reg_alpha': 0.7397217469766849, 'reg_lambda': 0.8165285963374457}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:20,044] Trial 28 finished with value: 0.2846330791414232 and parameters: {'max_depth': 35, 'n_estimators': 1422, 'learning_rate': 0.11179963735622846, 'colsample_bytree': 0.11363478052599521, 'num_leaves': 392, 'reg_alpha': 0.5586410823477588, 'reg_lambda': 0.49515982251640245}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:22,132] Trial 29 finished with value: 0.3161962853294594 and parameters: {'max_depth': 39, 'n_estimators': 894, 'learning_rate': 0.7067311567101416, 'colsample_bytree': 0.20264446543977876, 'num_leaves': 246, 'reg_alpha': 0.8277542424782391, 'reg_lambda': 0.38031998234809833}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:23,842] Trial 30 finished with value: 0.2902978693320266 and parameters: {'max_depth': 30, 'n_estimators': 1062, 'learning_rate': 0.4539489108677946, 'colsample_bytree': 0.5247804362214092, 'num_leaves': 302, 'reg_alpha': 0.5728984826286834, 'reg_lambda': 0.5707619370013504}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:29,421] Trial 31 finished with value: 0.27557587283956797 and parameters: {'max_depth': 26, 'n_estimators': 1153, 'learning_rate': 0.15548703685698181, 'colsample_bytree': 0.35755483384331116, 'num_leaves': 414, 'reg_alpha': 0.6557323234190832, 'reg_lambda': 0.6788309364364105}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:33,553] Trial 32 finished with value: 0.2861381279478822 and parameters: {'max_depth': 25, 'n_estimators': 1237, 'learning_rate': 0.21290587397072844, 'colsample_bytree': 0.33355824043254884, 'num_leaves': 465, 'reg_alpha': 0.6130009116598428, 'reg_lambda': 0.7271359519291707}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:39,990] Trial 33 finished with value: 0.27988738184804607 and parameters: {'max_depth': 20, 'n_estimators': 1097, 'learning_rate': 0.11241926036326264, 'colsample_bytree': 0.27261039697063993, 'num_leaves': 350, 'reg_alpha': 0.677988435219136, 'reg_lambda': 0.8485651159998069}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:45,815] Trial 34 finished with value: 0.2799319560685846 and parameters: {'max_depth': 28, 'n_estimators': 1349, 'learning_rate': 0.1591383421673421, 'colsample_bytree': 0.15012559467215575, 'num_leaves': 415, 'reg_alpha': 0.7458898936065202, 'reg_lambda': 0.53597101318694}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:48,419] Trial 35 finished with value: 0.28358820005314755 and parameters: {'max_depth': 31, 'n_estimators': 1191, 'learning_rate': 0.3258679891730658, 'colsample_bytree': 0.4382351348328016, 'num_leaves': 378, 'reg_alpha': 0.8183014229336567, 'reg_lambda': 0.6158926574625447}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:34:50,503] Trial 36 finished with value: 0.2779335207408836 and parameters: {'max_depth': 6, 'n_estimators': 987, 'learning_rate': 0.21912298941350394, 'colsample_bytree': 0.27765864277023455, 'num_leaves': 179, 'reg_alpha': 0.5694437876854713, 'reg_lambda': 0.7562061735286106}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:00,300] Trial 37 finished with value: 0.27562969702517925 and parameters: {'max_depth': 21, 'n_estimators': 1280, 'learning_rate': 0.10097242365568976, 'colsample_bytree': 0.2208065358320278, 'num_leaves': 458, 'reg_alpha': 0.45910908893169394, 'reg_lambda': 0.6611880870838136}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:02,975] Trial 38 finished with value: 0.28134479486951314 and parameters: {'max_depth': 34, 'n_estimators': 1587, 'learning_rate': 0.25375856348425097, 'colsample_bytree': 0.6357061625192505, 'num_leaves': 325, 'reg_alpha': 0.5086327566315614, 'reg_lambda': 0.8335541754275263}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:07,054] Trial 39 finished with value: 0.2772146382041118 and parameters: {'max_depth': 16, 'n_estimators': 837, 'learning_rate': 0.1733056879347039, 'colsample_bytree': 0.4064731403164977, 'num_leaves': 271, 'reg_alpha': 0.7121163405566502, 'reg_lambda': 0.8858637609568847}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:08,692] Trial 40 finished with value: 0.33426080223284593 and parameters: {'max_depth': 18, 'n_estimators': 1020, 'learning_rate': 0.8519070442327735, 'colsample_bytree': 0.48813974678535965, 'num_leaves': 362, 'reg_alpha': 0.6405731243337571, 'reg_lambda': 0.6461054460742309}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:15,195] Trial 41 finished with value: 0.27778596200301464 and parameters: {'max_depth': 26, 'n_estimators': 1161, 'learning_rate': 0.16983342178015545, 'colsample_bytree': 0.36341604581044706, 'num_leaves': 418, 'reg_alpha': 0.6577978713494343, 'reg_lambda': 0.683286296239889}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:22,370] Trial 42 finished with value: 0.2797798201330873 and parameters: {'max_depth': 24, 'n_estimators': 1117, 'learning_rate': 0.14555282792613936, 'colsample_bytree': 0.332642594015323, 'num_leaves': 404, 'reg_alpha': 0.5999904480268811, 'reg_lambda': 0.7605757766434458}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:26,304] Trial 43 finished with value: 0.28441463416046214 and parameters: {'max_depth': 27, 'n_estimators': 925, 'learning_rate': 0.2421602137297045, 'colsample_bytree': 0.35709670380481934, 'num_leaves': 500, 'reg_alpha': 0.5350302338322558, 'reg_lambda': 0.6831017124280541}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:28,171] Trial 44 finished with value: 0.30308756803296577 and parameters: {'max_depth': 22, 'n_estimators': 1275, 'learning_rate': 0.1027756356604776, 'colsample_bytree': 0.46313003369123473, 'num_leaves': 4, 'reg_alpha': 0.6657757487581447, 'reg_lambda': 0.6002140420105991}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:33,524] Trial 45 finished with value: 0.2831544108943555 and parameters: {'max_depth': 38, 'n_estimators': 1189, 'learning_rate': 0.149443054855772, 'colsample_bytree': 0.1950451625449444, 'num_leaves': 472, 'reg_alpha': 0.8530628169652117, 'reg_lambda': 0.5073298779564139}. Best is trial 14 with value: 0.2730527748346632.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:35:37,116] Trial 46 finished with value: 0.28694790255484437 and parameters: {'max_depth': 13, 'n_estimators': 1351, 'learning_rate': 0.33751367699090135, 'colsample_bytree': 0.2401594308102682, 'num_leaves': 383, 'reg_alpha': 0.7568753719260112, 'reg_lambda': 0.7742938514867925}. Best is trial 14 with value: 0.2730527748346632.\n",
      "[I 2020-10-24 12:35:43,784] Trial 47 finished with value: 0.2718213326536507 and parameters: {'max_depth': 30, 'n_estimators': 1054, 'learning_rate': 0.10417981242620984, 'colsample_bytree': 0.29385095448022197, 'num_leaves': 337, 'reg_alpha': 0.7991573833877426, 'reg_lambda': 0.6551007927065132}. Best is trial 47 with value: 0.2718213326536507.\n",
      "[I 2020-10-24 12:35:49,051] Trial 48 finished with value: 0.2712419066362349 and parameters: {'max_depth': 31, 'n_estimators': 768, 'learning_rate': 0.10395358602462655, 'colsample_bytree': 0.3004582369227073, 'num_leaves': 332, 'reg_alpha': 0.7969567856974819, 'reg_lambda': 0.6432689443285323}. Best is trial 48 with value: 0.2712419066362349.\n",
      "[I 2020-10-24 12:35:52,677] Trial 49 finished with value: 0.2858382840089473 and parameters: {'max_depth': 35, 'n_estimators': 749, 'learning_rate': 0.23529478992869876, 'colsample_bytree': 0.1352745458012939, 'num_leaves': 307, 'reg_alpha': 0.8066334657890399, 'reg_lambda': 0.6422687135944876}. Best is trial 48 with value: 0.2712419066362349.\n",
      "[I 2020-10-24 12:35:54,762] Trial 50 finished with value: 0.27972490144053425 and parameters: {'max_depth': 31, 'n_estimators': 787, 'learning_rate': 0.2910355834452195, 'colsample_bytree': 0.3002374655707375, 'num_leaves': 231, 'reg_alpha': 0.8686043844633025, 'reg_lambda': 0.5100689924994815}. Best is trial 48 with value: 0.2712419066362349.\n",
      "[I 2020-10-24 12:36:00,369] Trial 51 finished with value: 0.27174895231586926 and parameters: {'max_depth': 29, 'n_estimators': 884, 'learning_rate': 0.10210965448637291, 'colsample_bytree': 0.28732691378398106, 'num_leaves': 335, 'reg_alpha': 0.8032437039600857, 'reg_lambda': 0.5764733230835579}. Best is trial 48 with value: 0.2712419066362349.\n",
      "[I 2020-10-24 12:36:08,926] Trial 52 finished with value: 0.2711500634901012 and parameters: {'max_depth': 29, 'n_estimators': 868, 'learning_rate': 0.10130592168165514, 'colsample_bytree': 0.29840872430993026, 'num_leaves': 338, 'reg_alpha': 0.7919788672424208, 'reg_lambda': 0.5736739628502263}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:12,406] Trial 53 finished with value: 0.2759763631947277 and parameters: {'max_depth': 28, 'n_estimators': 873, 'learning_rate': 0.18220820823417974, 'colsample_bytree': 0.2950956753651676, 'num_leaves': 334, 'reg_alpha': 0.8982277123530168, 'reg_lambda': 0.5546941997059293}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:17,084] Trial 54 finished with value: 0.27492120593483865 and parameters: {'max_depth': 33, 'n_estimators': 721, 'learning_rate': 0.14745902202832542, 'colsample_bytree': 0.3980955409608512, 'num_leaves': 338, 'reg_alpha': 0.8036769079632723, 'reg_lambda': 0.4504948534291394}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:23,428] Trial 55 finished with value: 0.2816421082177221 and parameters: {'max_depth': 29, 'n_estimators': 818, 'learning_rate': 0.1365003474472435, 'colsample_bytree': 0.25107237327942544, 'num_leaves': 293, 'reg_alpha': 0.7838574970088467, 'reg_lambda': 0.40971041651419626}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:28,378] Trial 56 finished with value: 0.27948354524979485 and parameters: {'max_depth': 30, 'n_estimators': 922, 'learning_rate': 0.1921594098544664, 'colsample_bytree': 0.31827556691948045, 'num_leaves': 271, 'reg_alpha': 0.8530460354313458, 'reg_lambda': 0.6253534093332387}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:34,554] Trial 57 finished with value: 0.2813521860645097 and parameters: {'max_depth': 32, 'n_estimators': 768, 'learning_rate': 0.12756569907535295, 'colsample_bytree': 0.27096990976921453, 'num_leaves': 305, 'reg_alpha': 0.7356719304719149, 'reg_lambda': 0.5703195966516061}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:37,118] Trial 58 finished with value: 0.3090858777603663 and parameters: {'max_depth': 28, 'n_estimators': 849, 'learning_rate': 0.5804048543690206, 'colsample_bytree': 0.1920797223159985, 'num_leaves': 315, 'reg_alpha': 0.7839956743172869, 'reg_lambda': 0.8765435707891305}. Best is trial 52 with value: 0.2711500634901012.\n",
      "[I 2020-10-24 12:36:43,210] Trial 59 finished with value: 0.27176912351730115 and parameters: {'max_depth': 24, 'n_estimators': 990, 'learning_rate': 0.10217154767584909, 'colsample_bytree': 0.2918538320957773, 'num_leaves': 356, 'reg_alpha': 0.2229894995193255, 'reg_lambda': 0.61329162116016}. Best is trial 52 with value: 0.2711500634901012.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized LGBM RMSLE 0.2711500634901012\n",
      "Wall time: 4min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def create_model(trial):\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 2, 40)\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 700, 2000)\n",
    "    learning_rate = trial.suggest_uniform('learning_rate', 0.1, 1)\n",
    "    colsample_bytree = trial.suggest_uniform('colsample_bytree', 0.1, 0.9)\n",
    "    num_leaves = trial.suggest_int(\"num_leaves\", 2, 500)\n",
    "    #min_child_samples = trial.suggest_int('min_child_samples', 3, 200)\n",
    "    reg_alpha = trial.suggest_uniform(\"reg_alpha\", 0.1, 0.9)\n",
    "    reg_lambda = trial.suggest_uniform(\"reg_lambda\", 0.1, 0.9)\n",
    "    model = LGBMRegressor(\n",
    "        max_depth=max_depth,\n",
    "        n_estimators = n_estimators,\n",
    "        learning_rate=learning_rate, \n",
    "        colsample_bytree = colsample_bytree,\n",
    "        num_leaves=num_leaves, \n",
    "        reg_alpha = reg_alpha,\n",
    "        reg_lambda = reg_lambda,\n",
    "        #min_child_samples=min_child_samples,\n",
    "        random_state=1999,\n",
    "        n_jobs = -3\n",
    "    )\n",
    "    return model\n",
    "\n",
    "sampler = TPESampler(seed=0)\n",
    "def objective(trial):\n",
    "    model = create_model(trial)\n",
    "    model.fit(X_trn, y_trn, eval_set = [(X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "    preds = model.predict(X_val)\n",
    "    score = np.sqrt(mean_squared_error(y_val,preds))\n",
    "    return score\n",
    "\n",
    "study = optuna.create_study(direction=\"minimize\", sampler=sampler)\n",
    "study.optimize(objective, n_trials=60)\n",
    "\n",
    "lgb_params = study.best_params\n",
    "lgb_params['random_state'] = 1999\n",
    "lgb = LGBMRegressor(**lgb_params)\n",
    "lgb.fit(X_trn, y_trn, eval_set = [(X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "preds = lgb.predict(X_val)\n",
    "print('Optimized LGBM RMSLE', np.sqrt(mean_squared_error(y_val, preds)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:36:49,216] A new study created in memory with name: no-name-c2309e32-8278-43a2-85e9-073a8d0018b8\n",
      "[I 2020-10-24 12:36:51,558] Trial 0 finished with value: 0.3484254148843745 and parameters: {'max_depth': 12, 'n_estimators': 692, 'learning_rate': 0.6424870384644795, 'subsample': 0.5849460328672382, 'colsample_bytree': 0.4389238394711238, 'colsample_bylevel': 0.6167152904533248, 'reg_alpha': 5, 'reg_lambda': 8}. Best is trial 0 with value: 0.3484254148843745.\n",
      "[I 2020-10-24 12:36:53,231] Trial 1 finished with value: 0.39014295097313234 and parameters: {'max_depth': 13, 'n_estimators': 972, 'learning_rate': 0.9672964844509263, 'subsample': 0.4412629517549421, 'colsample_bytree': 0.7333800304661316, 'colsample_bylevel': 0.5231159358023236, 'reg_alpha': 9, 'reg_lambda': 2}. Best is trial 0 with value: 0.3484254148843745.\n",
      "[I 2020-10-24 12:36:59,233] Trial 2 finished with value: 0.3117214643345071 and parameters: {'max_depth': 12, 'n_estimators': 1037, 'learning_rate': 0.16393245237809825, 'subsample': 0.17754507673437125, 'colsample_bytree': 0.11617471795226059, 'colsample_bylevel': 0.7660958764383504, 'reg_alpha': 1, 'reg_lambda': 3}. Best is trial 2 with value: 0.3117214643345071.\n",
      "[I 2020-10-24 12:37:01,021] Trial 3 finished with value: 0.3578361838766217 and parameters: {'max_depth': 10, 'n_estimators': 1484, 'learning_rate': 0.8208196767816798, 'subsample': 0.5632249568005723, 'colsample_bytree': 0.6431036240951682, 'colsample_bylevel': 0.6765061237807334, 'reg_alpha': 2, 'reg_lambda': 10}. Best is trial 2 with value: 0.3117214643345071.\n",
      "[I 2020-10-24 12:37:08,012] Trial 4 finished with value: 0.2846797737398157 and parameters: {'max_depth': 7, 'n_estimators': 1043, 'learning_rate': 0.19531684646901293, 'subsample': 0.5215043732185252, 'colsample_bytree': 0.24906587466140798, 'colsample_bylevel': 0.6895345417031665, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 4 with value: 0.2846797737398157.\n",
      "[I 2020-10-24 12:37:09,777] Trial 5 finished with value: 0.3258810739179855 and parameters: {'max_depth': 7, 'n_estimators': 1432, 'learning_rate': 0.6558719473682894, 'subsample': 0.644765193222955, 'colsample_bytree': 0.5935471974998056, 'colsample_bylevel': 0.8549984628116993, 'reg_alpha': 10, 'reg_lambda': 9}. Best is trial 4 with value: 0.2846797737398157.\n",
      "[I 2020-10-24 12:37:16,797] Trial 6 finished with value: 0.27166962018692403 and parameters: {'max_depth': 8, 'n_estimators': 821, 'learning_rate': 0.1893523153230765, 'subsample': 0.9631300702946064, 'colsample_bytree': 0.6225120286383501, 'colsample_bylevel': 0.23672766810883614, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:18,142] Trial 7 finished with value: 0.35561684077853756 and parameters: {'max_depth': 10, 'n_estimators': 1065, 'learning_rate': 0.3925425061075173, 'subsample': 0.13419862956073392, 'colsample_bytree': 0.6074192463658681, 'colsample_bylevel': 0.8671594148996163, 'reg_alpha': 5, 'reg_lambda': 4}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:21,661] Trial 8 finished with value: 0.30956230881141933 and parameters: {'max_depth': 11, 'n_estimators': 680, 'learning_rate': 0.3279624422858039, 'subsample': 0.5150165878421126, 'colsample_bytree': 0.2955404736012822, 'colsample_bylevel': 0.22717566691641577, 'reg_alpha': 6, 'reg_lambda': 6}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:25,205] Trial 9 finished with value: 0.35073851690674845 and parameters: {'max_depth': 7, 'n_estimators': 805, 'learning_rate': 0.8005109338233182, 'subsample': 0.9451182375711604, 'colsample_bytree': 0.6300214935600355, 'colsample_bylevel': 0.11085730848968788, 'reg_alpha': 2, 'reg_lambda': 3}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:27,975] Trial 10 finished with value: 0.2979108839983118 and parameters: {'max_depth': 15, 'n_estimators': 542, 'learning_rate': 0.41965657626487457, 'subsample': 0.9273799159747689, 'colsample_bytree': 0.8594241151524176, 'colsample_bylevel': 0.3279836636513024, 'reg_alpha': 7, 'reg_lambda': 6}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:39,953] Trial 11 finished with value: 0.2724091835926307 and parameters: {'max_depth': 8, 'n_estimators': 1188, 'learning_rate': 0.11960269942202524, 'subsample': 0.7651761240077168, 'colsample_bytree': 0.36481590508040684, 'colsample_bylevel': 0.39576025555155325, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:47,880] Trial 12 finished with value: 0.27334230469971516 and parameters: {'max_depth': 9, 'n_estimators': 1274, 'learning_rate': 0.10036250778582735, 'subsample': 0.7875703447361717, 'colsample_bytree': 0.40962708483340043, 'colsample_bylevel': 0.39147144929227795, 'reg_alpha': 4, 'reg_lambda': 1}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:54,308] Trial 13 finished with value: 0.2905432294942634 and parameters: {'max_depth': 8, 'n_estimators': 1250, 'learning_rate': 0.27684567564329404, 'subsample': 0.7969319972484479, 'colsample_bytree': 0.320593823156597, 'colsample_bylevel': 0.17174077019162726, 'reg_alpha': 1, 'reg_lambda': 1}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:37:57,078] Trial 14 finished with value: 0.30287530724798706 and parameters: {'max_depth': 9, 'n_estimators': 884, 'learning_rate': 0.4866188974830228, 'subsample': 0.7994054979944962, 'colsample_bytree': 0.8237596050773424, 'colsample_bylevel': 0.3576151591411087, 'reg_alpha': 3, 'reg_lambda': 4}. Best is trial 6 with value: 0.27166962018692403.\n",
      "[I 2020-10-24 12:38:12,839] Trial 15 finished with value: 0.2696219422938147 and parameters: {'max_depth': 8, 'n_estimators': 1256, 'learning_rate': 0.11399224633135674, 'subsample': 0.979625827764785, 'colsample_bytree': 0.49782576502849285, 'colsample_bylevel': 0.4641310852439077, 'reg_alpha': 3, 'reg_lambda': 2}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:17,589] Trial 16 finished with value: 0.2754561595859257 and parameters: {'max_depth': 8, 'n_estimators': 1356, 'learning_rate': 0.2824434583520011, 'subsample': 0.9639320011361749, 'colsample_bytree': 0.501235404973802, 'colsample_bylevel': 0.49669804574814885, 'reg_alpha': 7, 'reg_lambda': 3}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:22,506] Trial 17 finished with value: 0.28162416888813885 and parameters: {'max_depth': 9, 'n_estimators': 814, 'learning_rate': 0.21817398240942637, 'subsample': 0.9870748796245933, 'colsample_bytree': 0.5235335854007063, 'colsample_bylevel': 0.27183814638536413, 'reg_alpha': 4, 'reg_lambda': 5}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:24,547] Trial 18 finished with value: 0.34874175917713335 and parameters: {'max_depth': 15, 'n_estimators': 512, 'learning_rate': 0.5695385821114881, 'subsample': 0.35436258785380104, 'colsample_bytree': 0.7343035394189916, 'colsample_bylevel': 0.501867305015588, 'reg_alpha': 2, 'reg_lambda': 2}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:31,094] Trial 19 finished with value: 0.27336459695201154 and parameters: {'max_depth': 10, 'n_estimators': 1152, 'learning_rate': 0.12301402523999222, 'subsample': 0.8872673930139577, 'colsample_bytree': 0.7297664123299715, 'colsample_bylevel': 0.17974782826704594, 'reg_alpha': 4, 'reg_lambda': 7}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:35,012] Trial 20 finished with value: 0.3023469777776291 and parameters: {'max_depth': 8, 'n_estimators': 933, 'learning_rate': 0.3471750519634768, 'subsample': 0.6855495964452467, 'colsample_bytree': 0.5249490843466151, 'colsample_bylevel': 0.10144690264316342, 'reg_alpha': 1, 'reg_lambda': 2}. Best is trial 15 with value: 0.2696219422938147.\n",
      "[I 2020-10-24 12:38:43,405] Trial 21 finished with value: 0.2681921717707495 and parameters: {'max_depth': 8, 'n_estimators': 1156, 'learning_rate': 0.1282423144462752, 'subsample': 0.8583044649709827, 'colsample_bytree': 0.39430648031413884, 'colsample_bylevel': 0.439284444843544, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:38:53,099] Trial 22 finished with value: 0.280191532970892 and parameters: {'max_depth': 7, 'n_estimators': 1158, 'learning_rate': 0.2232244970233237, 'subsample': 0.8691343416099941, 'colsample_bytree': 0.14154299147414773, 'colsample_bylevel': 0.5391103811452372, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:03,539] Trial 23 finished with value: 0.2723782085097788 and parameters: {'max_depth': 9, 'n_estimators': 1373, 'learning_rate': 0.10065514085399717, 'subsample': 0.9774030595467855, 'colsample_bytree': 0.2262841369521919, 'colsample_bylevel': 0.4383142669143582, 'reg_alpha': 2, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:10,116] Trial 24 finished with value: 0.276778055801978 and parameters: {'max_depth': 8, 'n_estimators': 1286, 'learning_rate': 0.2303414045048121, 'subsample': 0.8651437784721602, 'colsample_bytree': 0.42395162988174184, 'colsample_bylevel': 0.29885116051324345, 'reg_alpha': 4, 'reg_lambda': 4}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:18,121] Trial 25 finished with value: 0.2737130279545546 and parameters: {'max_depth': 7, 'n_estimators': 1109, 'learning_rate': 0.16627561996624757, 'subsample': 0.7249147292587833, 'colsample_bytree': 0.5579508564947471, 'colsample_bylevel': 0.42752325766709026, 'reg_alpha': 6, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:21,267] Trial 26 finished with value: 0.28339042801445086 and parameters: {'max_depth': 9, 'n_estimators': 661, 'learning_rate': 0.2878953115143035, 'subsample': 0.8598750890639935, 'colsample_bytree': 0.4410725361722996, 'colsample_bylevel': 0.5919471576791844, 'reg_alpha': 5, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:28,486] Trial 27 finished with value: 0.26851039571412016 and parameters: {'max_depth': 11, 'n_estimators': 855, 'learning_rate': 0.1080574794367903, 'subsample': 0.9258737576354611, 'colsample_bytree': 0.7026800236002454, 'colsample_bylevel': 0.45020286258033043, 'reg_alpha': 3, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:30,059] Trial 28 finished with value: 0.3320551001844118 and parameters: {'max_depth': 11, 'n_estimators': 1199, 'learning_rate': 0.45460309470803917, 'subsample': 0.30114781827189474, 'colsample_bytree': 0.688839756595227, 'colsample_bylevel': 0.45749949565291725, 'reg_alpha': 2, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:37,185] Trial 29 finished with value: 0.2732184755459941 and parameters: {'max_depth': 13, 'n_estimators': 918, 'learning_rate': 0.10344855793136895, 'subsample': 0.6204554647788604, 'colsample_bytree': 0.3734693164331789, 'colsample_bylevel': 0.6283217877414551, 'reg_alpha': 5, 'reg_lambda': 5}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:45,547] Trial 30 finished with value: 0.2687662532581637 and parameters: {'max_depth': 12, 'n_estimators': 1329, 'learning_rate': 0.10067225176673156, 'subsample': 0.9010792397620144, 'colsample_bytree': 0.4501213056757911, 'colsample_bylevel': 0.5993128190449555, 'reg_alpha': 4, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:51,630] Trial 31 finished with value: 0.27582648079042166 and parameters: {'max_depth': 12, 'n_estimators': 1333, 'learning_rate': 0.14379819568859178, 'subsample': 0.8997721677827899, 'colsample_bytree': 0.4606674149955082, 'colsample_bylevel': 0.5680100043473298, 'reg_alpha': 4, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:39:58,239] Trial 32 finished with value: 0.271011859033451 and parameters: {'max_depth': 13, 'n_estimators': 1414, 'learning_rate': 0.10354752803350614, 'subsample': 0.822024678847399, 'colsample_bytree': 0.46769902909358585, 'colsample_bylevel': 0.6398757075791928, 'reg_alpha': 3, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:02,664] Trial 33 finished with value: 0.2932758245752846 and parameters: {'max_depth': 14, 'n_estimators': 981, 'learning_rate': 0.26190715729541464, 'subsample': 0.9888689588685218, 'colsample_bytree': 0.36873509426457296, 'colsample_bylevel': 0.4616845352798998, 'reg_alpha': 4, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:08,014] Trial 34 finished with value: 0.2773452098803198 and parameters: {'max_depth': 12, 'n_estimators': 1239, 'learning_rate': 0.1660373271477356, 'subsample': 0.731272377910252, 'colsample_bytree': 0.5570630716888448, 'colsample_bylevel': 0.7479239240804452, 'reg_alpha': 1, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:09,932] Trial 35 finished with value: 0.38504726617757457 and parameters: {'max_depth': 12, 'n_estimators': 1092, 'learning_rate': 0.99982758777795, 'subsample': 0.9196335629118042, 'colsample_bytree': 0.301525758310862, 'colsample_bylevel': 0.5530808884442094, 'reg_alpha': 5, 'reg_lambda': 4}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:11,628] Trial 36 finished with value: 0.33527324087542265 and parameters: {'max_depth': 10, 'n_estimators': 1458, 'learning_rate': 0.6941563807302837, 'subsample': 0.8353918529289328, 'colsample_bytree': 0.48613063651931304, 'colsample_bylevel': 0.48004430282137, 'reg_alpha': 2, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:18,618] Trial 37 finished with value: 0.27061633352816955 and parameters: {'max_depth': 11, 'n_estimators': 1496, 'learning_rate': 0.1654178512424468, 'subsample': 0.9124634293262661, 'colsample_bytree': 0.6773947456943833, 'colsample_bylevel': 0.6959873814807831, 'reg_alpha': 3, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:25,014] Trial 38 finished with value: 0.2870120369259731 and parameters: {'max_depth': 11, 'n_estimators': 1023, 'learning_rate': 0.21155278578006254, 'subsample': 0.7447368485717474, 'colsample_bytree': 0.2058378449151346, 'colsample_bylevel': 0.4051891123623292, 'reg_alpha': 3, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:27,831] Trial 39 finished with value: 0.2920097006390113 and parameters: {'max_depth': 13, 'n_estimators': 1304, 'learning_rate': 0.34045382728924695, 'subsample': 0.6763350470229136, 'colsample_bytree': 0.7820076726946146, 'colsample_bylevel': 0.5183757531895644, 'reg_alpha': 2, 'reg_lambda': 10}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:32,399] Trial 40 finished with value: 0.2861408843764808 and parameters: {'max_depth': 12, 'n_estimators': 746, 'learning_rate': 0.1703520326478664, 'subsample': 0.46883890748781515, 'colsample_bytree': 0.570977006129526, 'colsample_bylevel': 0.3563302784131808, 'reg_alpha': 4, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:38,398] Trial 41 finished with value: 0.26980996636427307 and parameters: {'max_depth': 11, 'n_estimators': 1404, 'learning_rate': 0.14623666417075687, 'subsample': 0.9165585797102416, 'colsample_bytree': 0.6746085890006496, 'colsample_bylevel': 0.7177004571007245, 'reg_alpha': 3, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:46,397] Trial 42 finished with value: 0.26918048376305836 and parameters: {'max_depth': 10, 'n_estimators': 1398, 'learning_rate': 0.13861762740368577, 'subsample': 0.9482705314050377, 'colsample_bytree': 0.666679969677794, 'colsample_bylevel': 0.786440630267211, 'reg_alpha': 3, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:40:48,114] Trial 43 finished with value: 0.3566522604350323 and parameters: {'max_depth': 10, 'n_estimators': 1321, 'learning_rate': 0.9223743668001674, 'subsample': 0.9865910485060146, 'colsample_bytree': 0.7869765973900437, 'colsample_bylevel': 0.832551238356811, 'reg_alpha': 2, 'reg_lambda': 2}. Best is trial 21 with value: 0.2681921717707495.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:40:52,985] Trial 44 finished with value: 0.27951169656618036 and parameters: {'max_depth': 10, 'n_estimators': 1219, 'learning_rate': 0.2465995758376201, 'subsample': 0.9408529214823024, 'colsample_bytree': 0.42031961409098134, 'colsample_bylevel': 0.7819678687376859, 'reg_alpha': 5, 'reg_lambda': 1}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:41:01,103] Trial 45 finished with value: 0.26939303241529095 and parameters: {'max_depth': 10, 'n_estimators': 1399, 'learning_rate': 0.10084507800598488, 'subsample': 0.8473951465943335, 'colsample_bytree': 0.5912419067918427, 'colsample_bylevel': 0.8931271251942052, 'reg_alpha': 3, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:41:05,669] Trial 46 finished with value: 0.2782779708034813 and parameters: {'max_depth': 10, 'n_estimators': 1500, 'learning_rate': 0.1974721828618271, 'subsample': 0.8503415148024464, 'colsample_bytree': 0.6078716081527894, 'colsample_bylevel': 0.8953535017786367, 'reg_alpha': 4, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:41:09,445] Trial 47 finished with value: 0.28481217314790613 and parameters: {'max_depth': 12, 'n_estimators': 1454, 'learning_rate': 0.3040718510748419, 'subsample': 0.8273113180395468, 'colsample_bytree': 0.7333775506636758, 'colsample_bylevel': 0.8947016963455512, 'reg_alpha': 3, 'reg_lambda': 4}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:41:17,933] Trial 48 finished with value: 0.27175703104497256 and parameters: {'max_depth': 11, 'n_estimators': 1378, 'learning_rate': 0.10103511931791047, 'subsample': 0.7713419272484674, 'colsample_bytree': 0.6517044956896006, 'colsample_bylevel': 0.7967058571299107, 'reg_alpha': 1, 'reg_lambda': 5}. Best is trial 21 with value: 0.2681921717707495.\n",
      "[I 2020-10-24 12:41:20,752] Trial 49 finished with value: 0.28502121169460864 and parameters: {'max_depth': 10, 'n_estimators': 753, 'learning_rate': 0.3956718402995217, 'subsample': 0.8873365914015828, 'colsample_bytree': 0.5899855474827211, 'colsample_bylevel': 0.826331525714131, 'reg_alpha': 2, 'reg_lambda': 3}. Best is trial 21 with value: 0.2681921717707495.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized XGB RMSLE 0.27543876908577986\n",
      "Wall time: 4min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def create_model(trial):\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 7, 15)\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 500, 1500)\n",
    "    learning_rate = trial.suggest_uniform('learning_rate', 0.1, 1)\n",
    "    subsample = trial.suggest_uniform('subsample', 0.1, 0.99)\n",
    "    colsample_bytree = trial.suggest_uniform('colsample_bytree', 0.1, 0.9)\n",
    "    colsample_bylevel = trial.suggest_uniform('colsample_bylevel', 0.1, 0.9)\n",
    "    #num_leaves = trial.suggest_int(\"num_leaves\", 2, 5000)\n",
    "    #min_child_samples = trial.suggest_int('min_child_samples', 3, 200)\n",
    "    reg_alpha = trial.suggest_int(\"reg_alpha\", 1, 10)\n",
    "    reg_lambda = trial.suggest_int(\"reg_lambda\", 1, 10)\n",
    "    model = XGBRegressor(\n",
    "        max_depth = max_depth,\n",
    "        n_estimators = n_estimators,\n",
    "        learning_rate=learning_rate, \n",
    "        subsample = subsample,\n",
    "        colsample_bytree = colsample_bytree,\n",
    "        colsample_bylevel = colsample_bylevel,\n",
    "        #num_leaves=num_leaves, \n",
    "        #min_child_samples=min_child_samples,\n",
    "        random_state=0,\n",
    "        n_jobs = -3\n",
    "    )\n",
    "    return model\n",
    "\n",
    "sampler = TPESampler(seed=0)\n",
    "def objective(trial):\n",
    "    model = create_model(trial)\n",
    "    model.fit(X_trn, y_trn, eval_set = [ (X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "    preds = model.predict(X_val)\n",
    "    score = np.sqrt(mean_squared_error(y_val,preds))\n",
    "    return score\n",
    "\n",
    "study = optuna.create_study(direction=\"minimize\", sampler=sampler)\n",
    "study.optimize(objective, n_trials=50)\n",
    "\n",
    "xgb_params = study.best_params\n",
    "xgb_params['random_state'] = 0\n",
    "xgb = XGBRegressor(**xgb_params)\n",
    "xgb.fit(X_trn, y_trn, eval_set = [(X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "preds = xgb.predict(X_val)\n",
    "print('Optimized XGB RMSLE', np.sqrt(mean_squared_error(y_val, preds)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-10-24 12:41:33,482] A new study created in memory with name: no-name-25fe3206-0aab-4ffb-be3b-9d5ecbe6d9a3\n",
      "[I 2020-10-24 12:41:45,454] Trial 0 finished with value: 0.3019479905736397 and parameters: {'max_depth': 11, 'n_estimators': 1559, 'learning_rate': 0.7436704297351775, 'rsm': 0.636459404703763, 'reg_lambda': 4}. Best is trial 0 with value: 0.3019479905736397.\n",
      "[I 2020-10-24 12:41:57,073] Trial 1 finished with value: 0.2934580845103464 and parameters: {'max_depth': 8, 'n_estimators': 1723, 'learning_rate': 0.6813047017599905, 'rsm': 0.48945261802379636, 'reg_lambda': 7}. Best is trial 1 with value: 0.2934580845103464.\n",
      "[I 2020-10-24 12:42:16,268] Trial 2 finished with value: 0.2748916311581521 and parameters: {'max_depth': 7, 'n_estimators': 1600, 'learning_rate': 0.3453906651221019, 'rsm': 0.5251219544160014, 'reg_lambda': 8}. Best is trial 2 with value: 0.2748916311581521.\n",
      "[I 2020-10-24 12:43:45,235] Trial 3 finished with value: 0.2882239729912589 and parameters: {'max_depth': 13, 'n_estimators': 1600, 'learning_rate': 0.4535063164907468, 'rsm': 0.844110099548266, 'reg_lambda': 9}. Best is trial 2 with value: 0.2748916311581521.\n",
      "[I 2020-10-24 12:44:33,087] Trial 4 finished with value: 0.27296264499004946 and parameters: {'max_depth': 8, 'n_estimators': 1916, 'learning_rate': 0.11819655769629316, 'rsm': 0.8410316625376648, 'reg_lambda': 1}. Best is trial 4 with value: 0.27296264499004946.\n",
      "[I 2020-10-24 12:44:40,815] Trial 5 finished with value: 0.3058940179010183 and parameters: {'max_depth': 9, 'n_estimators': 1099, 'learning_rate': 0.8192427077950513, 'rsm': 0.5107166324051093, 'reg_lambda': 4}. Best is trial 4 with value: 0.27296264499004946.\n",
      "[I 2020-10-24 12:44:57,129] Trial 6 finished with value: 0.2912194361842667 and parameters: {'max_depth': 10, 'n_estimators': 1910, 'learning_rate': 0.7485693892533251, 'rsm': 0.6179976149468454, 'reg_lambda': 10}. Best is trial 4 with value: 0.27296264499004946.\n",
      "[I 2020-10-24 12:45:27,578] Trial 7 finished with value: 0.27147134154330876 and parameters: {'max_depth': 7, 'n_estimators': 1543, 'learning_rate': 0.19531684646901293, 'rsm': 0.5215043732185252, 'reg_lambda': 3}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:46:01,407] Trial 8 finished with value: 0.2800443588701951 and parameters: {'max_depth': 11, 'n_estimators': 1802, 'learning_rate': 0.22169635606490684, 'rsm': 0.3884854969359606, 'reg_lambda': 6}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:47:27,071] Trial 9 finished with value: 0.29731478171847897 and parameters: {'max_depth': 13, 'n_estimators': 1488, 'learning_rate': 0.6552405971872812, 'rsm': 0.9399357898780155, 'reg_lambda': 10}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:47:45,990] Trial 10 finished with value: 0.27151700269766216 and parameters: {'max_depth': 7, 'n_estimators': 1274, 'learning_rate': 0.11047367673850131, 'rsm': 0.13042662577121367, 'reg_lambda': 1}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:48:04,429] Trial 11 finished with value: 0.27283452091064053 and parameters: {'max_depth': 7, 'n_estimators': 1279, 'learning_rate': 0.10352259514661286, 'rsm': 0.13595040353731497, 'reg_lambda': 1}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:48:08,437] Trial 12 finished with value: 0.3250839091177902 and parameters: {'max_depth': 7, 'n_estimators': 1350, 'learning_rate': 0.9715702848411605, 'rsm': 0.12029943929684665, 'reg_lambda': 3}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:48:22,788] Trial 13 finished with value: 0.27463973463720565 and parameters: {'max_depth': 8, 'n_estimators': 1010, 'learning_rate': 0.2773074533165615, 'rsm': 0.28531175957553223, 'reg_lambda': 2}. Best is trial 7 with value: 0.27147134154330876.\n",
      "[I 2020-10-24 12:48:57,835] Trial 14 finished with value: 0.26997704355052254 and parameters: {'max_depth': 9, 'n_estimators': 1233, 'learning_rate': 0.11644235072217447, 'rsm': 0.25205222374124897, 'reg_lambda': 3}. Best is trial 14 with value: 0.26997704355052254.\n",
      "[I 2020-10-24 12:49:09,485] Trial 15 finished with value: 0.2802292020197442 and parameters: {'max_depth': 9, 'n_estimators': 1397, 'learning_rate': 0.4356656655606852, 'rsm': 0.27768471822157015, 'reg_lambda': 4}. Best is trial 14 with value: 0.26997704355052254.\n",
      "[I 2020-10-24 12:49:27,885] Trial 16 finished with value: 0.27622288280943913 and parameters: {'max_depth': 10, 'n_estimators': 1108, 'learning_rate': 0.2281015494699653, 'rsm': 0.2880611501459806, 'reg_lambda': 3}. Best is trial 14 with value: 0.26997704355052254.\n",
      "[I 2020-10-24 12:49:51,081] Trial 17 finished with value: 0.28495215910683597 and parameters: {'max_depth': 9, 'n_estimators': 1201, 'learning_rate': 0.5243780566860963, 'rsm': 0.7160239738412695, 'reg_lambda': 5}. Best is trial 14 with value: 0.26997704355052254.\n",
      "[I 2020-10-24 12:50:09,355] Trial 18 finished with value: 0.2863726531836051 and parameters: {'max_depth': 11, 'n_estimators': 1456, 'learning_rate': 0.3233438913049513, 'rsm': 0.3966768912477264, 'reg_lambda': 2}. Best is trial 14 with value: 0.26997704355052254.\n",
      "[I 2020-10-24 12:51:07,076] Trial 19 finished with value: 0.2749579664429738 and parameters: {'max_depth': 12, 'n_estimators': 1725, 'learning_rate': 0.16894689081887856, 'rsm': 0.40216984392036464, 'reg_lambda': 5}. Best is trial 14 with value: 0.26997704355052254.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized CAT RMLSE 0.27653013958429956\n",
      "Wall time: 9min 58s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def create_model(trial):\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 7, 13)\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 1000, 2000)\n",
    "    learning_rate = trial.suggest_uniform('learning_rate', 0.1, 1)\n",
    "    rsm = trial.suggest_uniform('rsm', 0.1, 0.99)\n",
    "    #num_leaves = trial.suggest_int(\"num_leaves\", 2, 5000)\n",
    "    #min_child_samples = trial.suggest_int('min_child_samples', 3, 200)\n",
    "    reg_lambda = trial.suggest_int(\"reg_lambda\", 1, 10)\n",
    "    model = CatBoostRegressor(\n",
    "        max_depth = max_depth,\n",
    "        n_estimators=n_estimators,\n",
    "        learning_rate=learning_rate,\n",
    "        rsm = rsm,\n",
    "        reg_lambda=reg_lambda,\n",
    "        cat_features=cat_features,\n",
    "        #num_leaves=num_leaves, \n",
    "        #min_child_samples=min_child_samples,\n",
    "        random_state=1999,\n",
    "        verbose=False\n",
    "    )\n",
    "    return model\n",
    "\n",
    "sampler = TPESampler(seed=0)\n",
    "def objective(trial):\n",
    "    model = create_model(trial)\n",
    "    model.fit(X_trn, y_trn, eval_set = [ (X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "    preds = model.predict(X_val)\n",
    "    score = np.sqrt(mean_squared_error(y_val,preds))\n",
    "    return score\n",
    "\n",
    "study = optuna.create_study(direction=\"minimize\", sampler=sampler)\n",
    "study.optimize(objective, n_trials=20)\n",
    "\n",
    "cat_params = study.best_params\n",
    "cat_params['random_state'] = 1999\n",
    "cat = CatBoostRegressor(**cat_params)\n",
    "cat.fit(X_trn, y_trn, eval_set = [ (X_val, y_val)], early_stopping_rounds = 50, verbose = False)\n",
    "preds = cat.predict(X_val)\n",
    "print('Optimized CAT RMLSE',np.sqrt(mean_squared_error(y_val, preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for lgb is 0.2711500634901012\n",
      "Error for rf is 0.28227506010457565\n",
      "Error for xgb is 0.27543876908577986\n",
      "Error for cat is 0.27653013958429956\n",
      "Wall time: 3min 34s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "models = {'lgb' : lgb, 'rf' : rf, 'xgb' : xgb, 'cat' : cat}\n",
    "\n",
    "for name,model in models.items():\n",
    "    error = predict(model, name)\n",
    "    print(f'Error for {name} is {error}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Error for lgb is 0.2715683553005204\n",
    "Error for rf is 0.2821737840690106\n",
    "Error for xgb is 0.2703749902764792\n",
    "Error for cat is 0.2730301229971128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# All Tuned Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. LGBM Models :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Manual Tuned\n",
    "lgb_1 = LGBMRegressor(random_state=1999,n_estimators=1000, learning_rate=0.13,num_leaves=70,max_depth=31,\n",
    "               reg_lambda=0.3, reg_alpha = 0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Optuna Tuned\n",
    "params = {'max_depth': 29, 'n_estimators': 868, 'learning_rate': 0.10130592168165514, 'colsample_bytree': 0.29840872430993026,\n",
    "          'num_leaves': 338, 'reg_alpha': 0.7919788672424208, 'reg_lambda': 0.5736739628502263}\n",
    "\n",
    "lgb_2 = LGBMRegressor(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 31, 'n_estimators': 768, 'learning_rate': 0.10395358602462655, 'colsample_bytree': 0.3004582369227073,\n",
    "          'num_leaves': 332, 'reg_alpha': 0.7969567856974819, 'reg_lambda': 0.6432689443285323}\n",
    "\n",
    "lgb_3 = LGBMRegressor(**params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### 2. XGBoost Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost\n",
    "params = {'max_depth': 9, 'n_estimators': 500,'learning_rate': 0.1, 'booster' : 'gbtree', 'n_jobs' : -1,\n",
    "         'subsample' : 0.9, 'colsample_bytree' : 0.8, 'colsample_bylevel' : 0.6, 'random_state' : 0}\n",
    "xgb_1 = XGBRegressor(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 8, 'n_estimators': 1156, 'learning_rate': 0.1282423144462752,\n",
    "          'subsample': 0.8583044649709827,'colsample_bytree': 0.39430648031413884,\n",
    "          'colsample_bylevel': 0.439284444843544,'random_state' : 0}\n",
    "\n",
    "xgb_2 = XGBRegressor(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 9, 'n_estimators': 1329, 'learning_rate': 0.10067225176673156, \n",
    "          'subsample': 0.9010792397620144, 'colsample_bytree': 0.4501213056757911, \n",
    "          'colsample_bylevel': 0.75993128190449555, 'random_state' : 0}\n",
    "xgb_3 = XGBRegressor(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost\n",
    "params = {'max_depth': 9, 'n_estimators': 2000,'learning_rate': 0.1, 'booster' : 'gbtree', 'n_jobs' : -1,\n",
    "         'subsample' : 0.9, 'colsample_bytree' : 0.8, 'colsample_bylevel' : 0.6, 'random_state' : 0}\n",
    "xgb_4 = XGBRegressor(**params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. RF Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 30, 'n_estimators': 2000, 'min_samples_split' : 2,'max_features' : 'sqrt', 'max_leaf_nodes' : 8000,\n",
    "          'bootstrap' : True,'random_state' : 1999, 'n_jobs' : -1}\n",
    "\n",
    "rf_1 = RandomForestRegressor(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 31, 'n_estimators': 1317, 'min_samples_split': 2, 'max_leaf_nodes': 6653, \n",
    "          'max_features': 0.6297197869507615, 'bootstrap' : True,'random_state' : 1999, 'n_jobs' : -1}\n",
    "\n",
    "rf_2 = RandomForestRegressor(**params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Catboost Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features = ['UNDER_CONSTRUCTION', 'RERA', 'BHK_NO.', 'BHK_OR_RK',\n",
    "                'READY_TO_MOVE', 'RESALE', 'City','Address']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'learning_rate' : 0.1,'max_depth' : 10,'n_estimators' : 2000,\n",
    "    'random_state' : 1999,'reg_lambda' : 6\n",
    "}\n",
    "cat_1 = CatBoostRegressor( **params,cat_features=cat_features,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'max_depth': 9, 'n_estimators': 1233, 'learning_rate': 0.11644235072217447, \n",
    "          'rsm': 0.25205222374124897, 'reg_lambda': 3}\n",
    "cat_2 = CatBoostRegressor( **params,cat_features=cat_features,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Bagging Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_1 = BaggingRegressor(base_estimator=lgb_1,n_estimators=20,max_samples=0.99,max_features=0.99,\n",
    "                       bootstrap=True,n_jobs=-1,random_state=0,verbose=0,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_2 = BaggingRegressor(base_estimator=lgb_2,n_estimators=20,max_samples=0.99,max_features=0.99,\n",
    "                       bootstrap=True,n_jobs=-1,random_state=0,verbose=0,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_3 = BaggingRegressor(base_estimator=xgb_1,n_estimators=20,max_samples=0.99,max_features=0.99,\n",
    "                       bootstrap=True,n_jobs=-1,random_state=0,verbose=0,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_4 = BaggingRegressor(base_estimator=xgb_3,n_estimators=20,max_samples=0.99,max_features=0.99,\n",
    "                       bootstrap=True,n_jobs=-1,random_state=0,verbose=0,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_5 = BaggingRegressor(base_estimator=xgb_4,n_estimators=20,max_samples=0.99,max_features=0.99,\n",
    "                       bootstrap=True,n_jobs=-1,random_state=0,verbose=0,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Predicting With All Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For Storing All OOFs and Preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new = train_proc[[target, 'RERA']].copy()\n",
    "test_new = test_proc[[target, 'RERA']].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "lgb_1_oofs, lgb_1_preds = cross_val(lgb_1, train_proc, test_proc, features, 'lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "lgb_2_oofs, lgb_2_preds = cross_val(lgb_2, train_proc, test_proc, features, 'lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "lgb_3_oofs, lgb_3_preds = cross_val(lgb_3, train_proc, test_proc, features, 'lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new['lgb_1'] = lgb_1_oofs\n",
    "test_new['lgb_1'] = lgb_1_preds\n",
    "\n",
    "train_new['lgb_2'] = lgb_2_oofs\n",
    "test_new['lgb_2'] = lgb_2_preds\n",
    "\n",
    "train_new['lgb_3'] = lgb_3_oofs\n",
    "test_new['lgb_3'] = lgb_3_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "xgb_1_oofs, xgb_1_preds = cross_val(xgb_1, train_proc, test_proc, features, 'xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "xgb_2_oofs, xgb_2_preds = cross_val(xgb_2, train_proc, test_proc, features, 'xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "xgb_3_oofs, xgb_3_preds = cross_val(xgb_3, train_proc, test_proc, features, 'xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "xgb_4_oofs, xgb_4_preds = cross_val(xgb_4, train_proc, test_proc, features, 'xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new['xgb_1'] = xgb_1_oofs\n",
    "test_new['xgb_1'] = xgb_1_preds\n",
    "\n",
    "train_new['xgb_2'] = xgb_2_oofs\n",
    "test_new['xgb_2'] = xgb_2_preds\n",
    "\n",
    "train_new['xgb_3'] = xgb_3_oofs\n",
    "test_new['xgb_3'] = xgb_3_preds\n",
    "\n",
    "train_new['xgb_4'] = xgb_4_oofs\n",
    "test_new['xgb_4'] = xgb_4_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "rf_1_oofs, rf_1_preds = normal_cross_val(rf_1, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "rf_2_oofs, rf_2_preds = normal_cross_val(rf_2, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new['rf_1'] = rf_1_oofs\n",
    "test_new['rf_1'] = rf_1_preds\n",
    "\n",
    "train_new['rf_2'] = rf_2_oofs\n",
    "test_new['rf_2'] = rf_2_preds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. CATBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "cat_1_oofs, cat_1_preds = cross_val(cat_1, train_proc, test_proc, features, 'cat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "cat_2_oofs, cat_2_preds = cross_val(cat_2, train_proc, test_proc, features, 'cat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new['cat_1'] = cat_1_oofs\n",
    "test_new['cat_1'] = cat_1_preds\n",
    "\n",
    "train_new['cat_2'] = cat_2_oofs\n",
    "test_new['cat_2'] = cat_2_preds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bag_1_oofs, bag_1_preds = normal_cross_val(bag_1, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bag_2_oofs, bag_2_preds = normal_cross_val(bag_2, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bag_3_oofs, bag_3_preds = normal_cross_val(bag_3, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bag_4_oofs, bag_4_preds = normal_cross_val(bag_4, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bag_5_oofs, bag_5_preds = normal_cross_val(bag_5, train_proc, test_proc, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_new['bag_1'] = bag_1_oofs\n",
    "test_new['bag_1'] = bag_1_preds\n",
    "\n",
    "train_new['bag_2'] = bag_2_oofs\n",
    "test_new['bag_2'] = bag_2_preds\n",
    "\n",
    "train_new['bag_3'] = bag_3_oofs\n",
    "test_new['bag_3'] = bag_3_preds\n",
    "\n",
    "train_new['bag_4'] = bag_4_oofs\n",
    "test_new['bag_4'] = bag_4_preds\n",
    "\n",
    "train_new['bag_5'] = bag_5_oofs\n",
    "test_new['bag_5'] = bag_5_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Level 1 Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_features = [c for c in train_new.columns if c not in [target, 'RERA']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "level_1_lgb_oofs, level_1_lgb_preds = cross_val(LGBMRegressor(), train_new, test_new, ens_features, 'lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "level_1_xgb_oofs, level_1_xgb_preds = cross_val(xgb_1, train_new, test_new, ens_features, 'xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "level_1_rf_1_oofs, level_1_rf_1_preds = normal_cross_val(rf_1, train_new, test_new, ens_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_ens = CatBoostRegressor(**params, verbose = False)\n",
    "\n",
    "level_1_cat_oofs, level_1_cat_preds = cross_val(cat_ens, train_new, test_new, ens_features, 'cat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "level_1_lgb_bag_oofs, level_1_lgb_bag_preds = normal_cross_val(BaggingRegressor(base_estimator = LGBMRegressor()), train_new, test_new, ens_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "level_1_xgb_bag_oofs, level_1_xgb_bag_preds = normal_cross_val(BaggingRegressor(base_estimator = xgb_1), train_new, test_new, ens_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "params = {'max_depth': 30, 'n_estimators': 1000, 'min_samples_split' : 2,'max_features' : 'sqrt', 'max_leaf_nodes' : 8000,\n",
    "          'bootstrap' : True,'random_state' : 1999, 'n_jobs' : -1}\n",
    "\n",
    "rf = RandomForestRegressor(**params)\n",
    "\n",
    "level_1_rf_bag_oofs, level_1_rf_bag_preds = normal_cross_val(BaggingRegressor(base_estimator = rf), train_new, test_new, ens_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_train_new = train_proc[[target, 'RERA']].copy()\n",
    "ens_test_new = test_proc[[target, 'RERA']].copy()\n",
    "\n",
    "ens_train_new['lgb'] = level_1_lgb_oofs\n",
    "ens_test_new['lgb'] = level_1_lgb_preds\n",
    "\n",
    "ens_train_new['xgb'] = level_1_xgb_oofs\n",
    "ens_test_new['xgb'] = level_1_xgb_preds\n",
    "\n",
    "ens_train_new['cat'] = level_1_cat_oofs\n",
    "ens_test_new['cat'] = level_1_cat_preds\n",
    "\n",
    "ens_train_new['rf'] = level_1_rf_1_oofs\n",
    "ens_test_new['rf'] = level_1_rf_1_preds\n",
    "\n",
    "ens_train_new['lgb_bag'] = level_1_lgb_bag_oofs\n",
    "ens_test_new['lgb_bag'] = level_1_lgb_bag_preds\n",
    "\n",
    "ens_train_new['rf1_bag'] = level_1_rf_bag_oofs\n",
    "ens_test_new['rf1_bag'] = level_1_rf_bag_preds\n",
    "\n",
    "ens_train_new['xgb_bag'] = level_1_xgb_bag_oofs\n",
    "ens_test_new['xgb_bag'] = level_1_xgb_bag_preds\n",
    "\n",
    "#ens_train_new['cat_bag'] = level_1_cat_bag_oofs\n",
    "#ens_test_new['cat_bag'] = level_1_cat_bag_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Level 2 Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_lvl_2_features = [c for c in ens_train_new.columns if c not in [target, 'RERA']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.25179921498193075\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2535219504056735\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2739281827903636\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2558337825788035\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.26326715456355954\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.2597971122339046\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression,Ridge,ARDRegression,SGDRegressor\n",
    "clf = LinearRegression()\n",
    "\n",
    "ens_linear_oofs, ens_linear_preds = normal_cross_val(clf, ens_train_new, ens_test_new, ens_lvl_2_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.25175475850941487\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2535203039104475\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2739348631855066\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.25583654710508374\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2632423637618381\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.25978510183717146\n"
     ]
    }
   ],
   "source": [
    "clf = Ridge()\n",
    "\n",
    "ens_ridge_oofs, ens_ridge_preds = normal_cross_val(clf, ens_train_new, ens_test_new, ens_lvl_2_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================Fold1===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2517881995349767\n",
      "\n",
      "================================Fold2===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.2535303430636621\n",
      "\n",
      "================================Fold3===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.27391173837391397\n",
      "\n",
      "================================Fold4===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.255989246938876\n",
      "\n",
      "================================Fold5===================================\n",
      "\n",
      " Root Log Mean Squared Error for Validation set is : 0.26326071963875886\n",
      "\n",
      "\\Root Log Mean Squared Error for oofs is 0.2598224685770336\n"
     ]
    }
   ],
   "source": [
    "clf = ARDRegression(normalize = True)\n",
    "\n",
    "ens_ARD_oofs, ens_ARD_preds = normal_cross_val(clf, ens_train_new, ens_test_new, ens_lvl_2_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = ens_linear_preds*0.60 + ens_ridge_preds*0.20 + ens_ARD_preds*0.20\n",
    "\n",
    "sample_sub['TARGET(PRICE_IN_LACS)']=np.abs((np.exp(preds)-1))\n",
    "sample_sub.to_csv(path + '\\\\Stacking.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
